Back to Article
dimension_tables.ipynb
Download Notebook 
In [18]:
import polars as pl
from IPython.display import HTML, display
from zfish.multi_table.tables_io import scan_resources
from great_tables import GT, md
from great_tables import loc, style
from zfish.plot.plot_commons import cmaps, lighten_hex_list
from pathlib import Path

TABLE_PATH = Path("../Tables")
r = scan_resources().collect()
In [19]:
roi_rename_map = {
    k: v
    for k, v in r.rois.sort("roi")
    .with_row_index()
    .with_columns(
        pl.concat_str(
            pl.lit("site"), pl.col("index").cast(pl.String).str.zfill(3), separator="_"
        ).alias("roi_short")
    )
    .select("roi", "roi_short")
    .rows()
}

df_channels = r.channels.filter(pl.col("contrast_limits").is_not_null())
df_channels_restain = r.channels.filter(pl.col("contrast_limits").is_null())
df_rois = (
    r.rois.sample(10, seed=42)
    .sort("roi")
    .with_columns(
        pl.concat_str(pl.lit("..."), pl.col("path").str.slice(47, 100)).alias("path")
    )
    .with_columns(pl.col("roi").cast(pl.String).replace(roi_rename_map))
    .select("roi", "path", "well", "well.z", "well.y", "well.x")
)
df_wells = r.wells.select(
    pl.col(
        "well",
        "row",
        "col",
        "is_control_well",
        "control_well_for_acquisition",
        "plate",
        "plate.z",
        "plate.y",
        "plate.x",
    ).sample(10, seed=42)
).rename(
    {"is_control_well": "is_control", "control_well_for_acquisition": "ctlr_for_acq"}
)


def plot_channel_table(df, title="channels"):
    return (
        GT(df)
        .tab_header(
            title=md(title),
        )
        .tab_style(
            style=style.fill(
                color=lighten_hex_list(cmaps.spatial_element_light, stops=2.4)[0]
            ),
            locations=loc.body(columns="c"),
        )
        .opt_vertical_padding(scale=0.4)
        .opt_horizontal_padding(scale=0.8)
        .cols_align("left", columns=["c", "stain"])
        .cols_align("right", columns=["acquisition", "wavelength", "contrast_limits"])
    )


def plot_table(
    df,
    title="rois",
    pk_column="roi",
    sub_color_col=None,
    color=cmaps.spatial_element_light[-1],
):
    gt = (
        GT(df)
        .tab_header(
            title=md(title),
        )
        .tab_style(
            style=style.fill(color=color),
            locations=loc.body(columns=pk_column),
        )
        .opt_vertical_padding(scale=0.4)
        .opt_horizontal_padding(scale=0.8)
    )
    if sub_color_col is not None:
        new_color = lighten_hex_list([color], stops=1.5)[0]
        return gt.tab_style(
            style=style.fill(color=new_color), locations=loc.body(columns=sub_color_col)
        )
    return gt


gt_channels = plot_channel_table(df_channels)

gt_channels_restain = plot_channel_table(df_channels_restain, title='channels_restain')

gt_rois = plot_table(df_rois, sub_color_col="well")

gt_wells = plot_table(
    df_wells, title="wells", pk_column="well", sub_color_col="plate"
)
In [20]:
Code 
gt_rois
rois
roi path well well.z well.y well.x
site_003 ...\imgs\B02_px+1825_py+1851.h5 B02 0.0 1851.0 1825.0
site_031 ...\imgs\C02_px+0211_py-0751.h5 C02 0.0 -751.0 211.0
site_032 ...\imgs\C02_px+0366_py+2039.h5 C02 0.0 2039.0 366.0
site_049 ...\imgs\C05_px-1262_py+1826.h5 C05 0.0 1826.0 -1262.0
site_060 ...\imgs\C07_px+2639_py-0215.h5 C07 0.0 -215.0 2639.0
site_062 ...\imgs\C07_px-1132_py+1322.h5 C07 0.0 1322.0 -1132.0
site_066 ...\imgs\D02_px+2103_py+0282.h5 D02 0.0 282.0 2103.0
site_096 ...\imgs\D07_px-1559_py+1432.h5 D07 0.0 1432.0 -1559.0
site_111 ...\imgs\E04_px+0120_py-2417.h5 E04 0.0 -2417.0 120.0
site_150 ...\imgs\F04_px+2038_py+1709.h5 F04 0.0 1709.0 2038.0 
In [21]:
Code 
gt_wells
wells
well row col is_control ctlr_for_acq plate plate.z plate.y plate.x
F06 F 06 false 1000 plate0 0.0 45000.0 54000.0
B07 B 07 true 0 plate0 0.0 9000.0 63000.0
G03 G 03 false 1000 plate0 0.0 54000.0 27000.0
G05 G 05 false 1000 plate0 0.0 54000.0 45000.0
C05 C 05 false 1000 plate0 0.0 18000.0 45000.0
D06 D 06 false 1000 plate0 0.0 27000.0 54000.0
G07 G 07 false 1000 plate0 0.0 54000.0 63000.0
D05 D 05 false 1000 plate0 0.0 27000.0 45000.0
B02 B 02 false 1000 plate0 0.0 9000.0 18000.0
B06 B 06 false 1000 plate0 0.0 9000.0 54000.0 
In [22]:
Code 
gt_channels
channels
c stain acquisition wavelength contrast_limits
DAPI.0 DAPI 0 405 [0.0, 550.0]
Pol-II-S2P.0 Pol-II-S2P 0 488 [0.0, 650.0]
FLAG.0 FLAG 0 561 [0.0, 400.0]
PCNA.0 PCNA 0 640 [0.0, 1500.0]
DAPI.1 DAPI 1 405 [0.0, 1000.0]
H3K27Ac.1 H3K27Ac 1 488 [30.0, 280.0]
bCatenin.1 bCatenin 1 561 [0.0, 350.0]
pH3.1 pH3 1 640 [0.0, 1200.0]
DAPI.2 DAPI 2 405 [0.0, 1500.0]
H2B.2 H2B 2 488 [0.0, 500.0]
ALYREF.2 ALYREF 2 561 [0.0, 560.0]
Pol-II-S5P.2 Pol-II-S5P 2 640 [0.0, 700.0]
DAPI.3 DAPI 3 405 [0.0, 800.0]
Nanog.3 Nanog 3 488 [0.0, 250.0]
YAP.3 YAP 3 561 [0.0, 800.0]
XRN2.3 XRN2 3 640 [0.0, 800.0] 
In [23]:
Code 
gt_channels_restain
channels_restain
c stain acquisition wavelength contrast_limits
H2B.3 H2B 3 488 None
H3K27Ac.3 H3K27Ac 3 488 None
Pol-II-S2P.3 Pol-II-S2P 3 488 None
FLAG.3 FLAG 3 561 None
bCatenin.3 bCatenin 3 561 None
ALYREF.3 ALYREF 3 561 None
pH3.3 pH3 3 640 None
PCNA.3 PCNA 3 640 None
Pol-II-S5P.3 Pol-II-S5P 3 640 None 
In [24]:
from zfish.multi_table.schemas_v2 import sel
import polars.selectors as cs
from itertools import product


df_label_obj = (
    r.label_objects.sample(16, seed=42)
    .join(r.multiscale_levels, on="m")
    .sort("o")
    .select(
        sel.idx - cs.by_name("m"),
        cs.starts_with("centroid"),
        *[
            (pl.col(f"m.{dim}.{bound}") * pl.col(f"scale.{dim}")).alias(
                f"bbx.{dim[1:]}.{bound}"
            )
            for dim, bound in product(["iz", "iy", "ix"], ["lower", "upper"])
        ],
    )
    .sample(16, seed=42)
    .sort("o", "roi")
    .with_columns(cs.starts_with("bbx").round(1))
    .select(
        sel.idx,
        cs.starts_with("centroid"),
        *[
            pl.concat_list(f"bbx.{dim}.lower", f"bbx.{dim}.upper").alias(f"bbx.{dim}")
            for dim in ["z", "y", "x"]
        ],
    )
)

centroid_cols = df_label_obj.select(cs.starts_with("centroid")).columns
bbx_cols = df_label_obj.select(cs.starts_with("bbx")).columns
bbx_nested = ["bbx.z", "bbx.y", "bbx.x"]


o_rename_map = {
    "nucleiRaw3": "nuc",
    "cells": "cell",
}
cmap_light = lighten_hex_list(cmaps.spatial_element_light, stops=2.4)
cmap_medium = lighten_hex_list(cmaps.spatial_element_light, stops=1.6)
In [25]:
r.object_types
shape: (8, 4)
o hierarchy_level is_primary parents
cat u8 bool list[cat]
"embryoRaw" 0 true []
"cells" 1 true ["embryoRaw"]
"nucleiRaw3" 2 true ["cells", "embryoRaw"]
"cyto" 2 true ["cells"]
"membrane" 2 true []
"nucleiRaw" 2 true []
"nucleiRaw2" 2 true []
"nucleiRaw4" 2 true [] 
In [26]:
from zfish.multi_table.tables_io import scan_resources
import polars as pl

r = scan_resources().collect()
df_object_counts = (
    r.label_objects.group_by("o")
    .agg(pl.len().alias("count"))
    .join(r.object_types, on="o", how='right')
    .select('o', 'count', 'hierarchy_level', 'parents')
    .sort("hierarchy_level", pl.col('count').is_null(), pl.col("parents").list.len())
)[:5]
(
    GT(df_object_counts)
    .cols_align("left")
    .tab_header(
        title=md("object types"),
    )
    .tab_style(
        style=style.fill(color=cmap_medium[1]),
        locations=loc.body(columns="o"),
    )
    .opt_vertical_padding(scale=0.4)
    .opt_horizontal_padding(scale=0.8)
)
object types
o count hierarchy_level parents
embryoRaw 212 0 []
cells 458246 1 ['embryoRaw']
cyto 458086 2 ['cells']
nucleiRaw3 459657 2 ['cells', 'embryoRaw']
membrane None 2 [] 
In [27]:
from zfish.multi_table.tables_io import scan_resources
import polars as pl

r = scan_resources().collect()
# df_object_counts = (
#     r.label_objects.group_by("o")
#     .agg(pl.len().alias('count'))
#     .join(r.object_types, on="o", how="left")
#     .sort("hierarchy_level", pl.col("parents").list.len())
# )


(
    GT(r.multiscale_levels)
    .cols_align("left")
    .tab_header(
        title=md("multiscale levels"),
    )
    .tab_style(
        style=style.fill(color=cmap_medium[3]),
        locations=loc.body(columns="m"),
    )
    .opt_vertical_padding(scale=0.4)
    .opt_horizontal_padding(scale=0.8)
)
multiscale levels
m scale.iz scale.iy scale.ix
0 1.0 0.325 0.325
1 1.0 0.65 0.65
2 1.0 1.3 1.3
3 2.0 2.6 2.6
4 5.0 5.2 5.2
5 10.0 10.4 10.4 
In [28]:
df_label_obj_gt = (
    GT(
        df_label_obj.with_columns(
            pl.col("roi").cast(pl.String).replace(roi_rename_map),
            # pl.col("o").cast(pl.String).replace(o_rename_map),
        )
    )
    .opt_vertical_padding(scale=0.4)
    .opt_horizontal_padding(scale=1.0)
    # .cols_align("left")
    .fmt_number(decimals=1, columns=centroid_cols)
    # .fmt_number(decimals=2, columns='bbx.y')
    .tab_spanner(label="label_object", columns=["o", "roi", "label"])
    .tab_spanner(label="centroid", columns=centroid_cols)
    .tab_spanner(label="bbx", columns=bbx_cols)
    .tab_style(
        style=style.fill(color=cmap_medium[1]),
        locations=loc.body(columns="o"),
    )
    .tab_style(
        style=style.fill(color=cmaps.spatial_element_light[-1]),
        locations=loc.body(columns="roi"),
    )
    .tab_style(
        style=style.fill(color=cmap_light[2]),
        locations=loc.body(columns="label"),
    )
    .cols_label(dict(zip(centroid_cols, [e.split(".")[-1] for e in centroid_cols])))
    .cols_label(dict(zip(bbx_cols, [e.split(".")[-1] for e in centroid_cols])))
    .tab_header(
        title=md("label objects"),
    )
)
# with pl.Config(tbl_rows=16):
In [29]:
df_images = (
    r.images.filter(
        pl.struct("m", "roi", "c").is_in(pl.struct("m", "roi", "c").head(5))
        | pl.struct("m", "roi", "c").is_in(pl.struct("m", "roi", "c").tail(5))
    )
    .with_columns(pl.col("roi").cast(pl.String).replace(roi_rename_map))
    .select(["m", "roi", "c", "c.path"])
)

gt_images = (
    GT(
        df_images.with_columns(
            pl.col("roi").cast(pl.String).replace(roi_rename_map),
            # pl.col("o").cast(pl.String).replace(o_rename_map),
        )
    )
    .opt_vertical_padding(scale=0.4)
    .opt_horizontal_padding(scale=1.0)
    .cols_align("left")
    # .fmt_number(decimals=1, columns=centroid_cols)
    # .fmt_number(decimals=2, columns='bbx.y')
    .tab_spanner(label="image", columns=["m", "roi", "c"])
    .tab_style(
        style=style.fill(color=cmap_medium[3]),
        locations=loc.body(columns="m"),
    )
    .tab_style(
        style=style.fill(color=cmaps.spatial_element_light[-1]),
        locations=loc.body(columns="roi"),
    )
    .tab_style(
        style=style.fill(color=cmap_medium[0]),
        locations=loc.body(columns="c"),
    )
    # .cols_label(dict(zip(centroid_cols, [e.split(".")[-1] for e in centroid_cols])))
    # .cols_label(dict(zip(bbx_cols, [e.split(".")[-1] for e in centroid_cols])))
    .tab_header(
        title=md("images"),
    )
)

df_label_images = (
    r.label_images.filter(
        pl.col("o").is_in(["nucleiRaw", "nucleiRaw2", "nucleiRaw4"]).not_()
    )
    .sort(["roi", "m", "o"])
    .filter(
        pl.struct("m", "roi", "o").is_in(pl.struct("m", "roi", "o").head(10))
        # | pl.struct("m", "roi", "o").is_in(pl.struct("m", "roi", "o").tail(5))
    )
    .with_columns(pl.col("roi").cast(pl.String).replace(roi_rename_map))
    .select(
        [
            "m",
            "roi",
            "o",
            "o.path",
            pl.col("resample_scale_factors").alias("m_scale_factors"),
        ]
    )
)

def plot_label_images_table(df_label_images, title='label images'):
    return (
        GT(
            df_label_images.with_columns(
                pl.col("roi").cast(pl.String).replace(roi_rename_map),
                # pl.col("o").cast(pl.String).replace(o_rename_map),
            )
        )
        .opt_vertical_padding(scale=0.4)
        .opt_horizontal_padding(scale=1.0)
        .cols_align("left")
        # .fmt_number(decimals=1, columns=centroid_cols)
        # .fmt_number(decimals=2, columns='bbx.y')
        .tab_spanner(label="label_image", columns=["m", "roi", "o"])
        .tab_style(
            style=style.fill(color=cmap_medium[3]),
            locations=loc.body(columns="m"),
        )
        .tab_style(
            style=style.fill(color=cmaps.spatial_element_light[-1]),
            locations=loc.body(columns="roi"),
        )
        .tab_style(
            style=style.fill(color=cmap_medium[1]),
            locations=loc.body(columns="o"),
        )
        # .cols_label(dict(zip(centroid_cols, [e.split(".")[-1] for e in centroid_cols])))
        # .cols_label(dict(zip(bbx_cols, [e.split(".")[-1] for e in centroid_cols])))
        .tab_header(
            title=md(title),
        )
    )

gt_label_images = plot_label_images_table(df_label_images)
In [30]:
Code 
gt_images
images
image c.path
m roi c
0 site_000 ALYREF.2 /ch_10/0
0 site_000 DAPI.0 /ch_00/0
0 site_000 DAPI.1 /ch_04/0
0 site_000 DAPI.2 /ch_08/0
0 site_000 DAPI.3 /ch_12/0
5 site_211 Pol-II-S2P.0 /ch_01/5
5 site_211 Pol-II-S5P.3 /ch_15/5
5 site_211 Pol-II-S5P.2 /ch_11/5
5 site_211 bCatenin.1 /ch_06/5
5 site_211 pH3.1 /ch_07/5 
In [31]:
Code 
gt_label_images
label images
label_image o.path m_scale_factors
m roi o
0 site_000 cells /lbl_cells [1.0, 2.0, 2.0]
0 site_000 cyto /lbl_cyto [1.0, 2.0, 2.0]
0 site_000 embryoRaw /lbl_embryo [1.0, 2.0, 2.0]
0 site_000 membrane /lbl_mem [1.0, 2.0, 2.0]
0 site_000 nucleiRaw3 /lbl_nuc_raw3 [1.0, 2.0, 2.0]
1 site_000 cells /lbl_cells None
1 site_000 cyto /lbl_cyto None
1 site_000 embryoRaw /lbl_embryo None
1 site_000 membrane /lbl_mem None
1 site_000 nucleiRaw3 /lbl_nuc_raw3 None 
In [32]:
Code 
df_label_obj_gt
label objects
label_object centroid bbx
o roi label z y x z y x
cells site_016 3177 194.5 531.2 457.7 [171.0, 211.0] [501.2, 550.6] [432.3, 477.1]
cells site_052 705 69.3 209.8 376.2 [53.0, 91.0] [197.6, 224.9] [364.0, 390.0]
cells site_090 4381 159.8 325.4 461.3 [141.0, 172.0] [310.7, 349.7] [448.5, 473.9]
cells site_114 2041 205.1 425.5 502.9 [178.0, 226.0] [395.2, 444.0] [487.5, 520.7]
cells site_119 99 36.7 390.9 253.3 [23.0, 54.0] [367.9, 406.9] [237.9, 276.3]
cyto site_016 2737 None None None [None, None] [None, None] [None, None]
cyto site_126 1266 None None None [None, None] [None, None] [None, None]
cyto site_157 2024 None None None [None, None] [None, None] [None, None]
nucleiRaw3 site_014 462 59.3 250.6 465.7 [50.0, 69.0] [246.4, 256.1] [460.9, 470.6]
nucleiRaw3 site_025 2534 143.0 271.5 324.0 [136.0, 151.0] [268.5, 275.6] [321.8, 327.6]
nucleiRaw3 site_041 2087 165.4 412.2 406.8 [160.0, 173.0] [408.2, 416.7] [403.0, 410.8]
nucleiRaw3 site_068 2952 137.8 338.3 514.3 [128.0, 148.0] [335.4, 341.9] [510.3, 518.7]
nucleiRaw3 site_072 194 45.6 306.9 289.7 [38.0, 55.0] [302.3, 312.0] [284.7, 295.1]
nucleiRaw3 site_129 228 71.7 273.2 207.4 [65.0, 79.0] [269.1, 277.6] [204.1, 211.3]
nucleiRaw3 site_164 1107 90.9 497.5 269.7 [83.0, 99.0] [493.4, 501.8] [265.2, 275.0]
nucleiRaw3 site_182 955 84.1 322.6 393.2 [75.0, 94.0] [318.5, 327.6] [388.7, 398.5] 
In [33]:
df_label_obj_gt = (
    GT(
    r.hierarchy.sample(10, seed=43).sort(
        ["o.parent", "label.parent"], descending=[True, False]
    ).with_columns(pl.col("roi").cast(pl.String).replace(roi_rename_map)))
    .opt_vertical_padding(scale=0.4)
    .opt_horizontal_padding(scale=1.0)
    .cols_align("left")
    # .fmt_number(decimals=1, columns=centroid_cols)
    # .fmt_number(decimals=2, columns='bbx.y')
    # .tab_spanner(label="centroid", columns=centroid_cols)
    # .tab_spanner(label="bbx", columns=bbx_cols)
    .tab_style(
        style=style.fill(color=cmap_medium[1]),
        locations=loc.body(columns=["o.parent", "o.child"]),
    )
    .tab_style(
        style=style.fill(color=cmaps.spatial_element_light[-1]),
        locations=loc.body(columns="roi"),
    )
    .tab_style(
        style=style.fill(color=cmap_light[2]),
        locations=loc.body(columns= ["label.parent", "label.child"]),
    )
    # .cols_label(dict(zip(centroid_cols, [e.split(".")[-1] for e in centroid_cols])))
    .cols_label({'label.parent': 'lbl.parent', 'label.child': 'lbl.child'})
    .cols_move_to_start(cs.ends_with('parent'))
    .tab_spanner(label="label_object.child", columns=["roi", "o.child", 'label.child'])
    .tab_spanner(label="label_object.parent", columns=["o.parent", 'label.parent', "roi"])


    .tab_header(
        title=md("hierarchy"),
    )
)
In [34]:
Code 
df_label_obj_gt
hierarchy
label_object.parent  
o.parent lbl.parent label_object.child
roi o.child lbl.child
embryoRaw 1 site_103 nucleiRaw3 4859
embryoRaw 1 site_207 cells 1528
embryoRaw 1 site_119 cells 1969
embryoRaw 1 site_008 nucleiRaw3 2466
embryoRaw 1 site_177 cells 1775
cells 42 site_014 cyto 42
cells 796 site_057 cyto 796
cells 1171 site_114 cyto 1171
cells 1937 site_051 nucleiRaw3 1937
cells 3046 site_017 cyto 3046